import asyncio

from com.arcfox.base.base_spider import ExecuteType
from com.arcfox.school.processor.rank.yidu_school_processor import YiDuSchoolProcessor
from com.arcfox.school.spider.yzy.yzy_base_spider import YZYBaseSpider
from com.arcfox.util import async_request as requests
from com.arcfox.util.util import random_sleep


class YiDuSchoolSpider(YZYBaseSpider):
    '''
    易度排名 http://m.eduranking.cn/
    '''

    def __init__(self):
        self.url = "https://uwf7de983aad7a717eb.youzy.cn/youzy.dms.basiclib.api.eduranking.cgr.search"
        self.start_page = 1
        self.processor = YiDuSchoolProcessor()
        self.years = [2018, 2019, 2020, 2021, 2022]

    def get_params(self, year, page_num):
        return '{"year":'+str(year)+',"keyword":"","features":[],"collegeProvinceCodes":[],"categories":[""],"pageIndex":' + str(page_num) + ',"pageSize":20}'

    def get_headers(self, params):
        return {
            "Host": "uwf7de983aad7a717eb.youzy.cn",
            "u-sign": self.get_u_sgin(params),
            "Content-Type": "application/json",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36",
            "Origin": "http://m.eduranking.cn",
            "Referer": "http://m.eduranking.cn/"
        }

    async def _pull_task(self):
        return None, ExecuteType.ONCE

    async def _crawl_by_task(self, task):
        for year in self.years:
            params = self.get_params(year, self.start_page)
            resp = await requests.post_with_session(url=self.url, data=params, headers=self.get_headers(params))
            if resp.code == 200:
                result_json = resp.json()
                total_count = result_json['result']["totalCount"]
                total_pages = total_count / 20 if total_count % 20 == 0 else total_count / 20 + 1
                for i in range(1, int(total_pages) + 1):
                    params_item = self.get_params(year, i)
                    resp_item = await requests.post_with_session(url=self.url, data=params_item,
                                                                 headers=self.get_headers(params_item))
                    if resp_item.code == 200:
                        await self.processor.parse_and_save_data(resp_item.json())
                        await random_sleep(0.2)


if __name__ == "__main__":
    asyncio.run(YiDuSchoolSpider().start_crawl())
